package com.example.myFirstWebmagic.demo;

import java.net.URI;
import java.util.ArrayList;
import java.util.List;





import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import com.example.myFirstWebmagic.util.UrlUtil;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
/**
 * @Description: 样例代码（抓取环保部数据中心的空气实时质量）
 * @param @param args   
 * @return void  
 * @throws
 * @author xiongyao
 * @date 2016-12-21
 */
public class AirHourProcessorDemo implements PageProcessor{

	
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String [] startUrl=new String[]{"http://datacenter.mep.gov.cn/report/air_daily/airDairyCityHourMain.jsp"};
		// TODO Auto-generated method stub
		Spider.create(new AirHourProcessorDemo())
        //从"https://github.com/code4craft"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo()).addPipeline(new FilePipeline("E:/项目工程/Webmagic-demo/spiderFileHour"))
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();
	}

	public Site getSite() {
		// TODO Auto-generated method stub
		return site;
	}

	
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		// TODO Auto-generated method stub
		/*第一步，获取所有的城市的当天空气质量查询的页面链接*/
		List<String> cityHrefs= page.getHtml().css("a.link").links().all();
		
		/*第二步，所有链接的中文参数转码*/
		List<String> gbCityHref=new ArrayList<String>();
		System.out.println(page.getUrl().match());
		System.out.println(page.getUrl());
		try {
			for(String cityHref:cityHrefs){
				gbCityHref.add(UrlUtil.encodeUrl("http://datacenter.mep.gov.cn/report/air_daily/airDairyReport_24hours_city.jsp?"+URI.create(cityHref).getQuery(), "gb2312"));
			}
		} catch (Exception e) {
			// TODO: handle exception
			e.printStackTrace();
		}
		//日期
		List<String> dates= page.getHtml().css("#report1 tbody tr td:nth-of-type(2)","text").all();
		//城市
		List<String> citys= page.getHtml().css("#report1 tbody tr td:nth-of-type(3)","text").all();
		//aqi指数
		List<String> aqis= page.getHtml().css("#report1 tbody tr td:nth-of-type(4)","text").all();
		//空气质量级别
		List<String> airLevels= page.getHtml().css("#report1 tbody tr td:nth-of-type(5)","text").all();
		//污染物
		List<String> populates= page.getHtml().css("#report1 tbody tr td:nth-of-type(6)","text").all();
		page.putField("citys", citys);
		page.putField("dates", dates);
		page.putField("aqis", aqis);
		page.putField("airLevels", airLevels);
		page.putField("populates", populates);
		
		page.addTargetRequests(gbCityHref);
	}

}
